import os
from sklearn.model_selection import train_test_split
import pandas as pd
import numpy as np


def rename():
    for (root, dirs, files) in os.walk("tianchi_datasets", topdown=False):
        for name in files:
            new_name = os.path.join(root, root.split('/')[-1] + '_' + name)
            print(os.path.join(root, "\t", name), new_name)
            os.rename(os.path.join(root, name), new_name)


def break_data(target, rate=0.2):
    for (root, dirs, files) in os.walk(target, topdown=False):
        for name in files:
            if "_" in name:
                print(name)
                origin_filename = os.path.join(root, name)
                origin_dataset = pd.read_csv(origin_filename, delimiter="\t", header=None)  # 加入参数
                train_data, test_data = train_test_split(origin_dataset, test_size=rate)
                train_filename = os.path.join(root, 'train.csv')
                dev_filename = os.path.join(root, 'dev.csv')
                train_data.to_csv(train_filename, index=False, sep="\t", header=None)
                test_data.to_csv(dev_filename, index=False, sep="\t", header=None)


# /home/livingbody/miniconda3/envs/paddle2/bin/python /home/livingbody/PycharmProjects/pythonProject/file_run.py
# TNEWS_rain.csv
# /home/livingbody/PycharmProjects/pythonProject/file_run.py:21: ParserWarning: Falling back to the 'python' engine because the 'c' engine does not support regex separators (separators > 1 char and different from '\s+' are interpreted as regex); you can avoid this warning by specifying engine='python'.
#   origin_dataset = pd.read_csv(origin_filename, sep="/t", header=None)  # 加入参数
# OCEMOTION_train.csv
# Traceback (most recent call last):
#   File "/home/livingbody/PycharmProjects/pythonProject/file_run.py", line 32, in <module>
#     break_data(target='tianchi_datasets', rate=0.2)
#   File "/home/livingbody/PycharmProjects/pythonProject/file_run.py", line 21, in break_data
#     origin_dataset = pd.read_csv(origin_filename, sep="/t", header=None)  # 加入参数
#   File "/home/livingbody/miniconda3/envs/paddle2/lib/python3.7/site-packages/pandas/io/parsers.py", line 688, in read_csv
#     return _read(filepath_or_buffer, kwds)
#   File "/home/livingbody/miniconda3/envs/paddle2/lib/python3.7/site-packages/pandas/io/parsers.py", line 460, in _read
#     data = parser.read(nrows)
#   File "/home/livingbody/miniconda3/envs/paddle2/lib/python3.7/site-packages/pandas/io/parsers.py", line 1198, in read
#     ret = self._engine.read(nrows)
#   File "/home/livingbody/miniconda3/envs/paddle2/lib/python3.7/site-packages/pandas/io/parsers.py", line 2585, in read
#     alldata = self._rows_to_cols(content)
#   File "/home/livingbody/miniconda3/envs/paddle2/lib/python3.7/site-packages/pandas/io/parsers.py", line 3237, in _rows_to_cols
#     self._alert_malformed(msg, row_num + 1)
#   File "/home/livingbody/miniconda3/envs/paddle2/lib/python3.7/site-packages/pandas/io/parsers.py", line 2998, in _alert_malformed
#     raise ParserError(msg)
# pandas.errors.ParserError: Expected 1 fields in line 14732, saw 2. Error could possibly be due to quotes being ignored when a multi-char delimiter is used.
#
# Process finished with exit code 1


if __name__ == '__main__':
    # rename()
    break_data(target='tianchi_datasets', rate=0.2)
